﻿/// -----------------------------------------------------------------------
/// (c) 2009 Microsoft Corporation -- All rights reserved
/// This code is licensed under the MS-PL
/// http://www.opensource.org/licenses/ms-pl.html
/// Courtesy of the Open Source Techology Center: http://port25.technet.com
/// -----------------------------------------------------------------------

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using gsToolkit;
using Sgml;

namespace WebWalker {
    internal class Request : IEquatable<Request> {
        internal Request(Uri baseURI, string requestURI) {
            uri = baseURI.MakeAbsolute(requestURI);
        }

        internal Request(Uri requestURI) {
            uri = requestURI;
        }

        internal Uri uri;
        internal XmlNodeList formFields;
        internal RequestType requestType = RequestType.GET;

        internal bool HasForm {
            get { return formFields != null; }
        }

        public bool Equals(Request other) {
            if(other.formFields == null && formFields == null)
                return (uri.Equals(other.uri) && requestType == other.requestType);
            if(other.formFields != null && formFields != null)
                return (uri.Equals(other.uri) && requestType == other.requestType && formFields.GetHashCode() == other.formFields.GetHashCode());
            return false;
        }

        public override bool Equals(Object other) {
            if( other as Request == null )
                return false;
            return Equals((Request) other);
        }

        public override int GetHashCode() {
            unchecked {
                int result = (uri != null ? uri.GetHashCode() : 0);
                result = (result*397) ^ (formFields != null ? formFields.GetHashCode() : 0);
                result = (result*397) ^ requestType.GetHashCode();
                return result;
            }
        }
    }

    class Walker {
        public string LogUrlsPath;
        public bool NoOutput;
        public List<string> UrlMask;
        public string OutputPath;
        public int MaxRequests = 100;

        // formdata is a list of strings representing key value pairs, to be applied on
        // pages that match the dictionary key.
        public List<ComplexOption> FormData;

        public bool SaveCookies;
        public string CookieFile;

        private Queue<Request> requestUrls = new Queue<Request>();
        private List<Request> completedRequests = new List<Request>();

        private Uri startUrl;

        public int Run(string beginURL) {
            startUrl = new Uri(beginURL);

            requestUrls.Enqueue(new Request(startUrl));

            return RunQueue();
        }

        private void LoadCookiesFromFile() {
            if(File.Exists(CookieFile)) {
                WebExtensions.Cookies.Clear();
                WebExtensions.Cookies.LoadCookiesFromNSFile(CookieFile);
            }
        }

        private void SaveCookiesToFile() {
            if(SaveCookies) {
                WebExtensions.Cookies.SaveCookiesToNSFile(CookieFile);
            }
        }

        private int RunQueue() {
            LoadCookiesFromFile();

            XmlDocument xmlDoc;
            string document;

            while( requestUrls.Count > 0 && MaxRequests > completedRequests.Count ) {
                Console.Write("Requests: Queued[{1}] Completed[{0}]         \r", completedRequests.Count, requestUrls.Count);
                Request rq = requestUrls.Dequeue();
                completedRequests.Add(rq);


                document = "";

                switch( rq.requestType ) {
                    case RequestType.GET:
                        document = rq.uri.Get();
                        break;
                    case RequestType.POST:
                        document = "";
                        break;
                }


                try {
                    using(SgmlReader reader = new SgmlReader() { InputStream = new StringReader(document) }) {
                        xmlDoc = new XmlDocument();
                        xmlDoc.Load(reader);

                        foreach(XmlNode node in xmlDoc.XPath("//link[@href] | //a[@href]")) {
                            AddUrlToQueue(new Request(rq.uri, node.Attributes["href"].Value.HtmlDecode()));
                        }

                        foreach(XmlNode node in xmlDoc.XPath("//script[@src] | //img[@src]") ) {
                            AddUrlToQueue(new Request(rq.uri, node.Attributes["src"].Value.HtmlDecode()));
                        }

                        /*
                        XmlNodeList forms = xmlDoc.XPath("//form[@action]");

                        foreach(XmlNode form in forms) {
                            AddUrlToQueue(new Request(rq.uri, form.Attributes["action"].Value) {
                                formFields = form.XPath(".//input | .//button | .//select | .//textarea")
                            });
                        } */

                    }
                }
                catch {
                    // if it's not an HTML document, don't even worry.
                }
            }
            SaveCookiesToFile();

            Console.WriteLine("Completed Requests:[{0}]", completedRequests.Count);

            if(!string.IsNullOrEmpty(LogUrlsPath))
                File.WriteAllLines(LogUrlsPath, completedRequests.GetUris());

            WebExtensions.Cookies.ExpireCookies();
            return 0;
        }

        private bool FilterURI(Uri uri) {
            if(!uri.DnsSafeHost.Equals( startUrl.DnsSafeHost ))
                return false; // no following to other domains.

            if( UrlMask != null ) {
                bool found = false;
                foreach( string rx in UrlMask )
                    if(Regex.Match(uri.PathAndQuery, rx).Success) {
                        found = true;
                        break;
                    }
                if(!found)
                    return false;
            }
            return true;
        }

            private void AddUrlToQueue(Request request) {

            if(completedRequests.Contains(request))
                return;  // don't go where we've already gone.

            if(FilterURI(request.uri)) {
                foreach(Request rq in requestUrls)
                    if(rq.uri.AbsoluteUri == request.uri.AbsoluteUri) {
                        if( request.HasForm == false && rq.HasForm == false )
                            return;// this has no form, and it's got a match. skip-it.
                    }
                requestUrls.Enqueue(request);
            }


        }

    }
}
